#!/usr/bin/python
# -*- coding:utf-8 -*-

from scrapy.spider import Spider
from scrapy.selector import Selector
from scrapy import log
import time
import sys
sys.path.append('D:\\PYTHON\\tutorial\\tutorial')
reload(sys)
sys.setdefaultencoding('utf-8')
from items import SHMZXItem
class SHMZXSpider(Spider):
    """爬取5858168标签"""
    #log.start("log",loglevel='INFO')
    name = "5858168"
    custom_settings = {'ITEM_PIPELINES':{'tutorial.pipelines.SHMZX_new_Pipeline':300,}}
    allowed_domains = ["5858168.com"]
    start_urls = []
    GetthisAll = False #开关 是否搜全部的发布求购项目 False 不搜 True 全部 正确下 做一次就够了，耗时久 by cq

    if GetthisAll:
        for  num in range(1,1000): #撑死1000页
            start_urls.append("http://www.5858168.com/buy/index.asp?page="+str(num))
    else:
        for  num in range(1,10): #比较笨的方法 观测这个网站一天撑死就3页求购数据 如果是只要当天的 爬前10页
            start_urls.append("http://www.5858168.com/buy/index.asp?page="+str(num))

    today=  time.strftime("%m-%d", time.localtime())
    if today[0]=='0': #会有03-27 这样的日趋 而这个网站月份 3-27 会有当前日期是03-27 而网站数据3-2 这样的日期也符合简单的过滤规则  by cq
        today = today[1:]

    def parse(self, response):
        sel = Selector(response)
        sites = sel.xpath('//div[@id="BuyList"]/dl')
        items = []
        for site in sites:
            item = SHMZXItem()
            currentDate =str(site.xpath('dt/font/text()').extract()[0].encode('utf-8')).replace("(","").replace(")","")
            #if currentDate in self.today or self.GetthisAll:
            title = str(site.xpath('dt/h1/a/text()').extract()[0].encode('utf-8'))
            link ="www.5858168.com"+ str(site.xpath('dt/h1/a/@href').extract()[0].encode('utf-8'))
            desc = str(site.xpath('dt/p/text()').extract()[0].encode('utf-8'))
            people = str(site.xpath('dt/ul/li/font/text()').extract()[0].encode('utf-8'))
            tel = str(site.xpath('dt/ul/li/text()').extract()[1].encode('utf-8')).replace("(","").replace(")","") #0 没东西 1才有电话
            item['title'] = title
            item['link'] =  link
            item['desc'] =  desc
            item['tel'] =   tel[2:]
            item['people'] = people
            item['currentDate'] = currentDate
            items.append(item)
        return items
